{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob \n",
    "import os \n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "\n",
    "import pandas as pd \n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import  utils\n",
    "\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = os.getcwd()\n",
    "use_cuda = torch.cuda.is_available()\n",
    "torch.manual_seed(123)\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "feature_size=2048\n",
    "cnn_feature_extractor=torchvision.models.resnet50(pretrained=True) #resnet50 fc is for 1000 calsses\n",
    "modules = list(cnn_feature_extractor.children())[:-1] # delete the last fc layer.\n",
    "cnn_feature_extractor = nn.Sequential(*modules).to(device)\n",
    "\n",
    "# set requires_grad to false\n",
    "for param in cnn_feature_extractor.parameters():\n",
    "    param.requires_grad = False\n",
    "#print(cnn_feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_domain='sketch' # change here \n",
    "#domain_name=['sketch','quickdraw','infograph','real']\n",
    "domain_name=['real']\n",
    "#domain_name.remove(target_domain)\n",
    "\n",
    "class_name=[file  for file in os.listdir(domain_name[0]) if file[-3:] !='csv' ]\n",
    "\n",
    "csv_name_train={name: pd.read_csv(cwd+'/'+name+'/'+name+ '_train.csv',index_col=0) for name in domain_name}\n",
    "csv_name_test={target_domain: pd.read_csv(cwd+'/'+target_domain+'/'+target_domain+ '_test.csv',index_col=0)}\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cvs=pd.concat(csv_name_train[key] for key in domain_name)\n",
    "test_csv=csv_name_test[target_domain]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "color_transform=transforms.Compose([\n",
    "                transforms.Resize((224, 224)),\n",
    "                transforms.ToTensor(),\n",
    "                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                     std=[0.229, 0.224, 0.225])\n",
    "            ])\n",
    "\n",
    "class finalset(Dataset):\n",
    "    def __init__(self, train_cvs,train, transform=None):\n",
    "        \"\"\" Intialize the MNIST dataset \"\"\"\n",
    "        self.images = None\n",
    "        self.labels = None\n",
    "        self.csv=train_cvs\n",
    "        self.filenames = list(train_cvs.index)\n",
    "        self.train=train\n",
    "        self.transform = transform\n",
    "        self.len = len(self.filenames)                      \n",
    "    def __getitem__(self, index):\n",
    "\n",
    "        \"\"\" Get a sample from the dataset \"\"\"\n",
    "        \n",
    "        image_fn=self.filenames[index]\n",
    "        image = Image.open(image_fn)\n",
    "    \n",
    "        if image.mode != 'RGB':\n",
    "            image = np.expand_dims(image, axis=2)\n",
    "            image=np.concatenate((image,image,image),axis=2)\n",
    "\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        \n",
    "        if self.train is True:\n",
    "            label=self.csv.loc[image_fn,'label']\n",
    "            return image,label,image_fn\n",
    "        if self.train is False:\n",
    "            return image, -1, image_fn\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len  \n",
    "\n",
    "train_data  =finalset(train_cvs,train=True,transform=color_transform)\n",
    "trainloader = DataLoader(train_data, batch_size=64,shuffle=True) \n",
    "\n",
    "test_data  =finalset(test_csv,train=True,transform=color_transform)\n",
    "testloader = DataLoader(test_data, batch_size=64,shuffle=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "batch_num=20\n",
    "def extract_feature(dalaloader):\n",
    "    train_list=[]\n",
    "    for batch_idx, data in enumerate(trainloader):\n",
    "        print(batch_idx)\n",
    "        if batch_idx >= batch_num:\n",
    "        #o get only 512 samples\n",
    "                break\n",
    "        input1, label1, filenames = data\n",
    "        input1= input1.cuda()\n",
    "        feature=cnn_feature_extractor(input1)\n",
    "        train_list.append(feature)\n",
    "    return train_list,label1,filenames\n",
    "train_list=extract_feature(trainloader)\n",
    "test_list=extract_feature(testloader)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"./train_features_resnet50.pkl\", \"wb\") as f:\n",
    "    pickle.dump(train_list, f)\n",
    "with open(\"./valid_features_resnet50.pkl\", \"wb\") as f:\n",
    "    pickle.dump(test_list, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open(\"./train_features_resnet50.pkl\", \"rb\") as f:\n",
    "#     train_list=pickle.load( f)\n",
    "# with open(\"./valid_features_resnet50.pkl\", \"rb\") as f:\n",
    "#     test_list=pickle.load( f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([[[[0.0056]],\n",
      "\n",
      "         [[1.0340]],\n",
      "\n",
      "         [[0.6207]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1004]],\n",
      "\n",
      "         [[0.1289]],\n",
      "\n",
      "         [[0.0769]]],\n",
      "\n",
      "\n",
      "        [[[0.1502]],\n",
      "\n",
      "         [[0.3957]],\n",
      "\n",
      "         [[0.5806]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.4907]],\n",
      "\n",
      "         [[0.9848]],\n",
      "\n",
      "         [[0.9342]]],\n",
      "\n",
      "\n",
      "        [[[0.6719]],\n",
      "\n",
      "         [[0.1203]],\n",
      "\n",
      "         [[0.7249]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1111]],\n",
      "\n",
      "         [[0.2760]],\n",
      "\n",
      "         [[0.4008]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[1.2414]],\n",
      "\n",
      "         [[0.7586]],\n",
      "\n",
      "         [[1.5587]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2289]],\n",
      "\n",
      "         [[0.2429]],\n",
      "\n",
      "         [[1.4113]]],\n",
      "\n",
      "\n",
      "        [[[0.0084]],\n",
      "\n",
      "         [[1.8643]],\n",
      "\n",
      "         [[0.5816]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0958]],\n",
      "\n",
      "         [[0.0270]],\n",
      "\n",
      "         [[0.0132]]],\n",
      "\n",
      "\n",
      "        [[[0.1280]],\n",
      "\n",
      "         [[0.0763]],\n",
      "\n",
      "         [[0.2170]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5059]],\n",
      "\n",
      "         [[0.8160]],\n",
      "\n",
      "         [[0.0866]]]], device='cuda:0'), tensor([[[[0.2717]],\n",
      "\n",
      "         [[0.4168]],\n",
      "\n",
      "         [[0.6184]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.4183]],\n",
      "\n",
      "         [[0.3671]],\n",
      "\n",
      "         [[0.0775]]],\n",
      "\n",
      "\n",
      "        [[[1.3755]],\n",
      "\n",
      "         [[0.3109]],\n",
      "\n",
      "         [[1.6935]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5303]],\n",
      "\n",
      "         [[0.4461]],\n",
      "\n",
      "         [[0.0865]]],\n",
      "\n",
      "\n",
      "        [[[0.0194]],\n",
      "\n",
      "         [[0.7001]],\n",
      "\n",
      "         [[0.0999]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0932]],\n",
      "\n",
      "         [[1.6971]],\n",
      "\n",
      "         [[0.1552]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.1658]],\n",
      "\n",
      "         [[1.0424]],\n",
      "\n",
      "         [[1.0853]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6325]],\n",
      "\n",
      "         [[0.2394]],\n",
      "\n",
      "         [[0.2582]]],\n",
      "\n",
      "\n",
      "        [[[0.2359]],\n",
      "\n",
      "         [[0.4951]],\n",
      "\n",
      "         [[0.6742]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2779]],\n",
      "\n",
      "         [[0.3791]],\n",
      "\n",
      "         [[0.1471]]],\n",
      "\n",
      "\n",
      "        [[[0.0345]],\n",
      "\n",
      "         [[0.3340]],\n",
      "\n",
      "         [[0.0926]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3531]],\n",
      "\n",
      "         [[0.1267]],\n",
      "\n",
      "         [[0.3105]]]], device='cuda:0'), tensor([[[[0.3275]],\n",
      "\n",
      "         [[0.2384]],\n",
      "\n",
      "         [[0.4741]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6356]],\n",
      "\n",
      "         [[0.2666]],\n",
      "\n",
      "         [[0.1898]]],\n",
      "\n",
      "\n",
      "        [[[0.3753]],\n",
      "\n",
      "         [[0.2345]],\n",
      "\n",
      "         [[1.3325]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1564]],\n",
      "\n",
      "         [[0.2301]],\n",
      "\n",
      "         [[0.2695]]],\n",
      "\n",
      "\n",
      "        [[[1.1023]],\n",
      "\n",
      "         [[1.3951]],\n",
      "\n",
      "         [[0.4418]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.8230]],\n",
      "\n",
      "         [[0.9297]],\n",
      "\n",
      "         [[1.0900]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.2471]],\n",
      "\n",
      "         [[1.1790]],\n",
      "\n",
      "         [[0.6029]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0298]],\n",
      "\n",
      "         [[0.2107]],\n",
      "\n",
      "         [[0.4519]]],\n",
      "\n",
      "\n",
      "        [[[1.1890]],\n",
      "\n",
      "         [[1.1803]],\n",
      "\n",
      "         [[1.0489]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0172]],\n",
      "\n",
      "         [[0.0091]],\n",
      "\n",
      "         [[0.0586]]],\n",
      "\n",
      "\n",
      "        [[[0.1011]],\n",
      "\n",
      "         [[0.1013]],\n",
      "\n",
      "         [[0.0853]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0601]],\n",
      "\n",
      "         [[1.3888]],\n",
      "\n",
      "         [[0.3223]]]], device='cuda:0'), tensor([[[[2.0714e+00]],\n",
      "\n",
      "         [[6.7741e-01]],\n",
      "\n",
      "         [[5.4504e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[7.1819e-01]],\n",
      "\n",
      "         [[1.5745e-01]],\n",
      "\n",
      "         [[1.6532e-01]]],\n",
      "\n",
      "\n",
      "        [[[9.6276e-01]],\n",
      "\n",
      "         [[1.0898e+00]],\n",
      "\n",
      "         [[1.4822e+00]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[8.7273e-01]],\n",
      "\n",
      "         [[7.1178e-01]],\n",
      "\n",
      "         [[7.2643e-01]]],\n",
      "\n",
      "\n",
      "        [[[7.4659e-01]],\n",
      "\n",
      "         [[1.5497e-01]],\n",
      "\n",
      "         [[1.1618e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[7.3976e-01]],\n",
      "\n",
      "         [[6.4767e-01]],\n",
      "\n",
      "         [[4.7611e-01]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[1.1862e-01]],\n",
      "\n",
      "         [[9.3323e-01]],\n",
      "\n",
      "         [[1.1109e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[8.8159e-01]],\n",
      "\n",
      "         [[4.3728e-02]],\n",
      "\n",
      "         [[4.2631e-01]]],\n",
      "\n",
      "\n",
      "        [[[1.0863e-01]],\n",
      "\n",
      "         [[3.5004e-01]],\n",
      "\n",
      "         [[5.8365e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[9.9967e-04]],\n",
      "\n",
      "         [[8.6307e-01]],\n",
      "\n",
      "         [[2.6601e-01]]],\n",
      "\n",
      "\n",
      "        [[[3.9022e-02]],\n",
      "\n",
      "         [[1.4483e+00]],\n",
      "\n",
      "         [[5.5486e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[3.0070e-01]],\n",
      "\n",
      "         [[2.3690e-01]],\n",
      "\n",
      "         [[1.5132e-01]]]], device='cuda:0'), tensor([[[[1.4166]],\n",
      "\n",
      "         [[0.3197]],\n",
      "\n",
      "         [[0.2345]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1474]],\n",
      "\n",
      "         [[0.2887]],\n",
      "\n",
      "         [[0.0148]]],\n",
      "\n",
      "\n",
      "        [[[0.0972]],\n",
      "\n",
      "         [[0.3322]],\n",
      "\n",
      "         [[0.1590]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1309]],\n",
      "\n",
      "         [[1.5793]],\n",
      "\n",
      "         [[0.5869]]],\n",
      "\n",
      "\n",
      "        [[[0.1661]],\n",
      "\n",
      "         [[0.6005]],\n",
      "\n",
      "         [[0.7279]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1587]],\n",
      "\n",
      "         [[1.0340]],\n",
      "\n",
      "         [[0.9809]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.2307]],\n",
      "\n",
      "         [[0.3566]],\n",
      "\n",
      "         [[0.7339]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5148]],\n",
      "\n",
      "         [[1.0819]],\n",
      "\n",
      "         [[0.5894]]],\n",
      "\n",
      "\n",
      "        [[[0.7668]],\n",
      "\n",
      "         [[0.9049]],\n",
      "\n",
      "         [[0.0588]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1882]],\n",
      "\n",
      "         [[0.6907]],\n",
      "\n",
      "         [[0.7261]]],\n",
      "\n",
      "\n",
      "        [[[0.2147]],\n",
      "\n",
      "         [[0.0561]],\n",
      "\n",
      "         [[0.7399]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.4432]],\n",
      "\n",
      "         [[0.6057]],\n",
      "\n",
      "         [[0.7870]]]], device='cuda:0'), tensor([[[[0.3902]],\n",
      "\n",
      "         [[0.4428]],\n",
      "\n",
      "         [[0.5340]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1025]],\n",
      "\n",
      "         [[0.4895]],\n",
      "\n",
      "         [[0.2787]]],\n",
      "\n",
      "\n",
      "        [[[0.4689]],\n",
      "\n",
      "         [[0.3297]],\n",
      "\n",
      "         [[0.3766]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1709]],\n",
      "\n",
      "         [[0.4677]],\n",
      "\n",
      "         [[0.3771]]],\n",
      "\n",
      "\n",
      "        [[[0.1050]],\n",
      "\n",
      "         [[0.3319]],\n",
      "\n",
      "         [[0.2858]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0836]],\n",
      "\n",
      "         [[0.1437]],\n",
      "\n",
      "         [[0.0000]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.5501]],\n",
      "\n",
      "         [[0.6034]],\n",
      "\n",
      "         [[0.8766]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2309]],\n",
      "\n",
      "         [[0.1624]],\n",
      "\n",
      "         [[0.1649]]],\n",
      "\n",
      "\n",
      "        [[[0.0071]],\n",
      "\n",
      "         [[0.4184]],\n",
      "\n",
      "         [[0.0146]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0039]],\n",
      "\n",
      "         [[0.1838]],\n",
      "\n",
      "         [[0.0726]]],\n",
      "\n",
      "\n",
      "        [[[0.1475]],\n",
      "\n",
      "         [[0.5875]],\n",
      "\n",
      "         [[0.0277]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0237]],\n",
      "\n",
      "         [[0.0929]],\n",
      "\n",
      "         [[0.7608]]]], device='cuda:0'), tensor([[[[0.0832]],\n",
      "\n",
      "         [[0.7267]],\n",
      "\n",
      "         [[0.0886]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2911]],\n",
      "\n",
      "         [[0.1397]],\n",
      "\n",
      "         [[0.3798]]],\n",
      "\n",
      "\n",
      "        [[[0.2945]],\n",
      "\n",
      "         [[0.7485]],\n",
      "\n",
      "         [[0.4353]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1804]],\n",
      "\n",
      "         [[0.7791]],\n",
      "\n",
      "         [[0.4291]]],\n",
      "\n",
      "\n",
      "        [[[0.1527]],\n",
      "\n",
      "         [[0.3044]],\n",
      "\n",
      "         [[0.2859]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3206]],\n",
      "\n",
      "         [[0.1598]],\n",
      "\n",
      "         [[0.2650]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.3823]],\n",
      "\n",
      "         [[0.0320]],\n",
      "\n",
      "         [[0.2086]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1209]],\n",
      "\n",
      "         [[0.1172]],\n",
      "\n",
      "         [[0.9991]]],\n",
      "\n",
      "\n",
      "        [[[0.1332]],\n",
      "\n",
      "         [[0.1425]],\n",
      "\n",
      "         [[0.1025]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3959]],\n",
      "\n",
      "         [[0.3822]],\n",
      "\n",
      "         [[0.7675]]],\n",
      "\n",
      "\n",
      "        [[[0.0275]],\n",
      "\n",
      "         [[0.2222]],\n",
      "\n",
      "         [[0.4277]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0604]],\n",
      "\n",
      "         [[0.6366]],\n",
      "\n",
      "         [[0.2230]]]], device='cuda:0'), tensor([[[[0.4114]],\n",
      "\n",
      "         [[0.2148]],\n",
      "\n",
      "         [[0.1712]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3723]],\n",
      "\n",
      "         [[0.5575]],\n",
      "\n",
      "         [[0.1437]]],\n",
      "\n",
      "\n",
      "        [[[0.2104]],\n",
      "\n",
      "         [[0.2452]],\n",
      "\n",
      "         [[0.4136]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1417]],\n",
      "\n",
      "         [[0.2229]],\n",
      "\n",
      "         [[0.0908]]],\n",
      "\n",
      "\n",
      "        [[[0.1132]],\n",
      "\n",
      "         [[1.8915]],\n",
      "\n",
      "         [[1.0398]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2870]],\n",
      "\n",
      "         [[0.0173]],\n",
      "\n",
      "         [[0.2443]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.2565]],\n",
      "\n",
      "         [[0.2949]],\n",
      "\n",
      "         [[0.3431]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2309]],\n",
      "\n",
      "         [[0.2478]],\n",
      "\n",
      "         [[0.0949]]],\n",
      "\n",
      "\n",
      "        [[[0.0239]],\n",
      "\n",
      "         [[0.3376]],\n",
      "\n",
      "         [[0.3578]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0037]],\n",
      "\n",
      "         [[0.0000]],\n",
      "\n",
      "         [[0.2202]]],\n",
      "\n",
      "\n",
      "        [[[0.1702]],\n",
      "\n",
      "         [[0.3732]],\n",
      "\n",
      "         [[1.7399]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0832]],\n",
      "\n",
      "         [[0.3019]],\n",
      "\n",
      "         [[0.3367]]]], device='cuda:0'), tensor([[[[0.0219]],\n",
      "\n",
      "         [[0.3509]],\n",
      "\n",
      "         [[0.5566]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.8738]],\n",
      "\n",
      "         [[0.1893]],\n",
      "\n",
      "         [[0.0597]]],\n",
      "\n",
      "\n",
      "        [[[0.3613]],\n",
      "\n",
      "         [[0.0762]],\n",
      "\n",
      "         [[0.9013]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6806]],\n",
      "\n",
      "         [[0.3769]],\n",
      "\n",
      "         [[0.5345]]],\n",
      "\n",
      "\n",
      "        [[[0.1879]],\n",
      "\n",
      "         [[0.4964]],\n",
      "\n",
      "         [[0.4214]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0097]],\n",
      "\n",
      "         [[0.4431]],\n",
      "\n",
      "         [[0.3946]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.8533]],\n",
      "\n",
      "         [[0.9301]],\n",
      "\n",
      "         [[0.0929]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2854]],\n",
      "\n",
      "         [[1.0287]],\n",
      "\n",
      "         [[1.0356]]],\n",
      "\n",
      "\n",
      "        [[[1.0838]],\n",
      "\n",
      "         [[0.3424]],\n",
      "\n",
      "         [[0.2243]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0128]],\n",
      "\n",
      "         [[0.1316]],\n",
      "\n",
      "         [[0.4006]]],\n",
      "\n",
      "\n",
      "        [[[0.3500]],\n",
      "\n",
      "         [[0.1089]],\n",
      "\n",
      "         [[0.4144]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5099]],\n",
      "\n",
      "         [[1.1480]],\n",
      "\n",
      "         [[0.0000]]]], device='cuda:0'), tensor([[[[0.1975]],\n",
      "\n",
      "         [[0.1803]],\n",
      "\n",
      "         [[0.1289]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.5107]],\n",
      "\n",
      "         [[1.1684]],\n",
      "\n",
      "         [[0.3693]]],\n",
      "\n",
      "\n",
      "        [[[0.1994]],\n",
      "\n",
      "         [[1.5598]],\n",
      "\n",
      "         [[0.3671]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1698]],\n",
      "\n",
      "         [[0.0893]],\n",
      "\n",
      "         [[0.7448]]],\n",
      "\n",
      "\n",
      "        [[[0.4259]],\n",
      "\n",
      "         [[0.3870]],\n",
      "\n",
      "         [[0.1506]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3034]],\n",
      "\n",
      "         [[0.0714]],\n",
      "\n",
      "         [[0.2800]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.0156]],\n",
      "\n",
      "         [[0.1500]],\n",
      "\n",
      "         [[0.4525]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2236]],\n",
      "\n",
      "         [[0.2939]],\n",
      "\n",
      "         [[0.0688]]],\n",
      "\n",
      "\n",
      "        [[[0.1021]],\n",
      "\n",
      "         [[0.3477]],\n",
      "\n",
      "         [[0.6059]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3209]],\n",
      "\n",
      "         [[0.3762]],\n",
      "\n",
      "         [[0.0482]]],\n",
      "\n",
      "\n",
      "        [[[0.1577]],\n",
      "\n",
      "         [[1.3991]],\n",
      "\n",
      "         [[0.2850]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.9056]],\n",
      "\n",
      "         [[0.4656]],\n",
      "\n",
      "         [[0.1427]]]], device='cuda:0'), tensor([[[[0.0907]],\n",
      "\n",
      "         [[0.6315]],\n",
      "\n",
      "         [[1.6592]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0796]],\n",
      "\n",
      "         [[0.0966]],\n",
      "\n",
      "         [[0.0425]]],\n",
      "\n",
      "\n",
      "        [[[1.4085]],\n",
      "\n",
      "         [[0.1528]],\n",
      "\n",
      "         [[0.4321]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0535]],\n",
      "\n",
      "         [[0.3465]],\n",
      "\n",
      "         [[0.1205]]],\n",
      "\n",
      "\n",
      "        [[[0.0087]],\n",
      "\n",
      "         [[0.3667]],\n",
      "\n",
      "         [[0.2575]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1088]],\n",
      "\n",
      "         [[1.0117]],\n",
      "\n",
      "         [[0.5082]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.1297]],\n",
      "\n",
      "         [[0.3178]],\n",
      "\n",
      "         [[0.5059]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.4609]],\n",
      "\n",
      "         [[0.1764]],\n",
      "\n",
      "         [[0.3035]]],\n",
      "\n",
      "\n",
      "        [[[2.4285]],\n",
      "\n",
      "         [[0.6737]],\n",
      "\n",
      "         [[0.3004]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0411]],\n",
      "\n",
      "         [[0.3856]],\n",
      "\n",
      "         [[0.7102]]],\n",
      "\n",
      "\n",
      "        [[[0.4713]],\n",
      "\n",
      "         [[0.0602]],\n",
      "\n",
      "         [[0.2251]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.7071]],\n",
      "\n",
      "         [[0.1967]],\n",
      "\n",
      "         [[0.0336]]]], device='cuda:0'), tensor([[[[1.0352e-01]],\n",
      "\n",
      "         [[8.0491e-02]],\n",
      "\n",
      "         [[4.2177e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.3191e-01]],\n",
      "\n",
      "         [[5.4565e-01]],\n",
      "\n",
      "         [[4.7891e-01]]],\n",
      "\n",
      "\n",
      "        [[[2.4904e+00]],\n",
      "\n",
      "         [[1.5027e-01]],\n",
      "\n",
      "         [[1.0353e+00]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[6.3197e-03]],\n",
      "\n",
      "         [[4.1825e-01]],\n",
      "\n",
      "         [[2.1162e-01]]],\n",
      "\n",
      "\n",
      "        [[[2.4492e-01]],\n",
      "\n",
      "         [[4.8411e-01]],\n",
      "\n",
      "         [[6.6405e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[6.8004e-02]],\n",
      "\n",
      "         [[2.9364e-01]],\n",
      "\n",
      "         [[3.6350e-01]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[2.1514e-03]],\n",
      "\n",
      "         [[6.9223e-01]],\n",
      "\n",
      "         [[9.5141e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[4.0826e-01]],\n",
      "\n",
      "         [[3.5278e-01]],\n",
      "\n",
      "         [[2.2078e-01]]],\n",
      "\n",
      "\n",
      "        [[[4.7697e-01]],\n",
      "\n",
      "         [[1.1010e+00]],\n",
      "\n",
      "         [[1.5035e+00]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[3.8355e-01]],\n",
      "\n",
      "         [[4.1942e-01]],\n",
      "\n",
      "         [[1.3482e-01]]],\n",
      "\n",
      "\n",
      "        [[[1.7728e-01]],\n",
      "\n",
      "         [[5.2539e-01]],\n",
      "\n",
      "         [[2.5469e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[2.9554e-01]],\n",
      "\n",
      "         [[4.6774e-01]],\n",
      "\n",
      "         [[2.5091e-01]]]], device='cuda:0'), tensor([[[[1.3055]],\n",
      "\n",
      "         [[0.7042]],\n",
      "\n",
      "         [[0.8081]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5924]],\n",
      "\n",
      "         [[0.6259]],\n",
      "\n",
      "         [[1.0539]]],\n",
      "\n",
      "\n",
      "        [[[0.2763]],\n",
      "\n",
      "         [[1.1821]],\n",
      "\n",
      "         [[0.4570]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1885]],\n",
      "\n",
      "         [[0.1707]],\n",
      "\n",
      "         [[0.8582]]],\n",
      "\n",
      "\n",
      "        [[[0.1398]],\n",
      "\n",
      "         [[0.3799]],\n",
      "\n",
      "         [[0.3408]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2314]],\n",
      "\n",
      "         [[0.4964]],\n",
      "\n",
      "         [[0.1815]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.2301]],\n",
      "\n",
      "         [[0.1106]],\n",
      "\n",
      "         [[0.3800]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2819]],\n",
      "\n",
      "         [[0.4710]],\n",
      "\n",
      "         [[0.5505]]],\n",
      "\n",
      "\n",
      "        [[[1.0115]],\n",
      "\n",
      "         [[0.4690]],\n",
      "\n",
      "         [[1.4175]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2803]],\n",
      "\n",
      "         [[0.4371]],\n",
      "\n",
      "         [[0.1933]]],\n",
      "\n",
      "\n",
      "        [[[0.4186]],\n",
      "\n",
      "         [[0.7572]],\n",
      "\n",
      "         [[1.1310]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2120]],\n",
      "\n",
      "         [[1.1195]],\n",
      "\n",
      "         [[0.2736]]]], device='cuda:0'), tensor([[[[0.0000]],\n",
      "\n",
      "         [[0.6117]],\n",
      "\n",
      "         [[0.4173]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6392]],\n",
      "\n",
      "         [[0.1433]],\n",
      "\n",
      "         [[0.3639]]],\n",
      "\n",
      "\n",
      "        [[[0.2681]],\n",
      "\n",
      "         [[0.1739]],\n",
      "\n",
      "         [[0.1391]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1213]],\n",
      "\n",
      "         [[0.0497]],\n",
      "\n",
      "         [[0.3618]]],\n",
      "\n",
      "\n",
      "        [[[0.3689]],\n",
      "\n",
      "         [[1.3403]],\n",
      "\n",
      "         [[0.4145]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1171]],\n",
      "\n",
      "         [[0.2426]],\n",
      "\n",
      "         [[0.4592]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.4748]],\n",
      "\n",
      "         [[0.5483]],\n",
      "\n",
      "         [[0.4336]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1287]],\n",
      "\n",
      "         [[0.0306]],\n",
      "\n",
      "         [[0.2739]]],\n",
      "\n",
      "\n",
      "        [[[0.7662]],\n",
      "\n",
      "         [[1.3933]],\n",
      "\n",
      "         [[0.2001]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3847]],\n",
      "\n",
      "         [[0.2143]],\n",
      "\n",
      "         [[0.3245]]],\n",
      "\n",
      "\n",
      "        [[[0.0311]],\n",
      "\n",
      "         [[0.1829]],\n",
      "\n",
      "         [[0.2127]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1035]],\n",
      "\n",
      "         [[0.1052]],\n",
      "\n",
      "         [[0.0842]]]], device='cuda:0'), tensor([[[[0.2170]],\n",
      "\n",
      "         [[0.2251]],\n",
      "\n",
      "         [[0.3184]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1638]],\n",
      "\n",
      "         [[0.0890]],\n",
      "\n",
      "         [[0.0264]]],\n",
      "\n",
      "\n",
      "        [[[0.0439]],\n",
      "\n",
      "         [[0.8203]],\n",
      "\n",
      "         [[0.0929]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.4280]],\n",
      "\n",
      "         [[0.3440]],\n",
      "\n",
      "         [[0.0362]]],\n",
      "\n",
      "\n",
      "        [[[0.2224]],\n",
      "\n",
      "         [[0.0574]],\n",
      "\n",
      "         [[0.7585]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0937]],\n",
      "\n",
      "         [[0.3934]],\n",
      "\n",
      "         [[0.0483]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.1403]],\n",
      "\n",
      "         [[0.3332]],\n",
      "\n",
      "         [[0.0404]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2448]],\n",
      "\n",
      "         [[0.0031]],\n",
      "\n",
      "         [[0.0386]]],\n",
      "\n",
      "\n",
      "        [[[0.2999]],\n",
      "\n",
      "         [[0.4623]],\n",
      "\n",
      "         [[0.4461]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3931]],\n",
      "\n",
      "         [[1.0297]],\n",
      "\n",
      "         [[0.3482]]],\n",
      "\n",
      "\n",
      "        [[[0.5797]],\n",
      "\n",
      "         [[0.7344]],\n",
      "\n",
      "         [[0.4161]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0946]],\n",
      "\n",
      "         [[0.0965]],\n",
      "\n",
      "         [[0.5966]]]], device='cuda:0'), tensor([[[[0.0000]],\n",
      "\n",
      "         [[0.2243]],\n",
      "\n",
      "         [[0.0748]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3973]],\n",
      "\n",
      "         [[0.4153]],\n",
      "\n",
      "         [[0.1193]]],\n",
      "\n",
      "\n",
      "        [[[0.5881]],\n",
      "\n",
      "         [[0.7882]],\n",
      "\n",
      "         [[0.2101]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.7060]],\n",
      "\n",
      "         [[0.1392]],\n",
      "\n",
      "         [[0.9792]]],\n",
      "\n",
      "\n",
      "        [[[0.1257]],\n",
      "\n",
      "         [[0.1575]],\n",
      "\n",
      "         [[0.4202]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0287]],\n",
      "\n",
      "         [[0.9024]],\n",
      "\n",
      "         [[0.2893]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.2548]],\n",
      "\n",
      "         [[0.0244]],\n",
      "\n",
      "         [[1.0242]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0939]],\n",
      "\n",
      "         [[0.0919]],\n",
      "\n",
      "         [[0.7288]]],\n",
      "\n",
      "\n",
      "        [[[0.1498]],\n",
      "\n",
      "         [[0.5540]],\n",
      "\n",
      "         [[0.5568]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5469]],\n",
      "\n",
      "         [[0.0330]],\n",
      "\n",
      "         [[0.6972]]],\n",
      "\n",
      "\n",
      "        [[[1.0070]],\n",
      "\n",
      "         [[0.9619]],\n",
      "\n",
      "         [[0.7740]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6613]],\n",
      "\n",
      "         [[1.6105]],\n",
      "\n",
      "         [[0.1744]]]], device='cuda:0'), tensor([[[[1.5408e-01]],\n",
      "\n",
      "         [[3.2241e-01]],\n",
      "\n",
      "         [[1.6886e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[8.6822e-02]],\n",
      "\n",
      "         [[3.4011e-01]],\n",
      "\n",
      "         [[3.9627e-01]]],\n",
      "\n",
      "\n",
      "        [[[7.6143e-01]],\n",
      "\n",
      "         [[1.5690e+00]],\n",
      "\n",
      "         [[6.0302e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[4.2874e-01]],\n",
      "\n",
      "         [[8.6606e-02]],\n",
      "\n",
      "         [[1.0563e-01]]],\n",
      "\n",
      "\n",
      "        [[[2.7582e-01]],\n",
      "\n",
      "         [[3.1702e-01]],\n",
      "\n",
      "         [[3.8295e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[6.6381e-02]],\n",
      "\n",
      "         [[2.4654e-01]],\n",
      "\n",
      "         [[3.1915e-01]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[6.2567e-01]],\n",
      "\n",
      "         [[5.7931e-02]],\n",
      "\n",
      "         [[4.2572e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.1525e+00]],\n",
      "\n",
      "         [[4.2773e-01]],\n",
      "\n",
      "         [[5.4308e-04]]],\n",
      "\n",
      "\n",
      "        [[[2.3319e-03]],\n",
      "\n",
      "         [[3.5678e-03]],\n",
      "\n",
      "         [[2.1469e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.1821e-01]],\n",
      "\n",
      "         [[4.1936e-02]],\n",
      "\n",
      "         [[2.8259e-02]]],\n",
      "\n",
      "\n",
      "        [[[3.9653e-01]],\n",
      "\n",
      "         [[5.9987e-01]],\n",
      "\n",
      "         [[1.1798e-01]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[8.4372e-01]],\n",
      "\n",
      "         [[2.4601e-01]],\n",
      "\n",
      "         [[8.7077e-01]]]], device='cuda:0'), tensor([[[[2.6130]],\n",
      "\n",
      "         [[0.9891]],\n",
      "\n",
      "         [[3.6242]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1834]],\n",
      "\n",
      "         [[0.0942]],\n",
      "\n",
      "         [[1.4335]]],\n",
      "\n",
      "\n",
      "        [[[0.4352]],\n",
      "\n",
      "         [[0.4926]],\n",
      "\n",
      "         [[0.1180]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3827]],\n",
      "\n",
      "         [[0.4016]],\n",
      "\n",
      "         [[0.1914]]],\n",
      "\n",
      "\n",
      "        [[[0.0494]],\n",
      "\n",
      "         [[0.2666]],\n",
      "\n",
      "         [[0.2311]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.5192]],\n",
      "\n",
      "         [[0.0591]],\n",
      "\n",
      "         [[0.1916]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.0363]],\n",
      "\n",
      "         [[1.9881]],\n",
      "\n",
      "         [[0.3441]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2860]],\n",
      "\n",
      "         [[0.1871]],\n",
      "\n",
      "         [[0.4127]]],\n",
      "\n",
      "\n",
      "        [[[0.2325]],\n",
      "\n",
      "         [[0.8059]],\n",
      "\n",
      "         [[0.2580]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2276]],\n",
      "\n",
      "         [[0.0909]],\n",
      "\n",
      "         [[0.4483]]],\n",
      "\n",
      "\n",
      "        [[[0.0596]],\n",
      "\n",
      "         [[0.3856]],\n",
      "\n",
      "         [[0.2375]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0482]],\n",
      "\n",
      "         [[0.1229]],\n",
      "\n",
      "         [[0.0989]]]], device='cuda:0'), tensor([[[[0.0502]],\n",
      "\n",
      "         [[0.1816]],\n",
      "\n",
      "         [[0.1596]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1876]],\n",
      "\n",
      "         [[0.9371]],\n",
      "\n",
      "         [[0.0890]]],\n",
      "\n",
      "\n",
      "        [[[0.8221]],\n",
      "\n",
      "         [[0.1606]],\n",
      "\n",
      "         [[0.2312]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0420]],\n",
      "\n",
      "         [[0.8208]],\n",
      "\n",
      "         [[0.5148]]],\n",
      "\n",
      "\n",
      "        [[[0.1510]],\n",
      "\n",
      "         [[1.5757]],\n",
      "\n",
      "         [[0.0899]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.6316]],\n",
      "\n",
      "         [[1.5530]],\n",
      "\n",
      "         [[0.2792]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.1143]],\n",
      "\n",
      "         [[0.4360]],\n",
      "\n",
      "         [[0.3644]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1064]],\n",
      "\n",
      "         [[0.1365]],\n",
      "\n",
      "         [[0.0584]]],\n",
      "\n",
      "\n",
      "        [[[0.2009]],\n",
      "\n",
      "         [[0.5039]],\n",
      "\n",
      "         [[0.8805]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.1354]],\n",
      "\n",
      "         [[0.1976]],\n",
      "\n",
      "         [[0.0628]]],\n",
      "\n",
      "\n",
      "        [[[0.9500]],\n",
      "\n",
      "         [[0.5923]],\n",
      "\n",
      "         [[0.4990]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0790]],\n",
      "\n",
      "         [[0.1040]],\n",
      "\n",
      "         [[0.2949]]]], device='cuda:0'), tensor([[[[0.5363]],\n",
      "\n",
      "         [[0.3576]],\n",
      "\n",
      "         [[0.1543]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.1321]],\n",
      "\n",
      "         [[0.2744]],\n",
      "\n",
      "         [[0.3306]]],\n",
      "\n",
      "\n",
      "        [[[0.9706]],\n",
      "\n",
      "         [[0.6982]],\n",
      "\n",
      "         [[0.5706]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.3343]],\n",
      "\n",
      "         [[0.8855]],\n",
      "\n",
      "         [[1.0293]]],\n",
      "\n",
      "\n",
      "        [[[1.1062]],\n",
      "\n",
      "         [[0.1254]],\n",
      "\n",
      "         [[0.3606]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.2587]],\n",
      "\n",
      "         [[1.0288]],\n",
      "\n",
      "         [[0.1690]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[0.8635]],\n",
      "\n",
      "         [[0.2880]],\n",
      "\n",
      "         [[0.7049]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0133]],\n",
      "\n",
      "         [[0.2625]],\n",
      "\n",
      "         [[0.4194]]],\n",
      "\n",
      "\n",
      "        [[[0.4064]],\n",
      "\n",
      "         [[0.3133]],\n",
      "\n",
      "         [[0.9818]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[0.0113]],\n",
      "\n",
      "         [[0.4053]],\n",
      "\n",
      "         [[0.4360]]],\n",
      "\n",
      "\n",
      "        [[[0.0595]],\n",
      "\n",
      "         [[0.5872]],\n",
      "\n",
      "         [[0.2633]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[1.0338]],\n",
      "\n",
      "         [[0.2688]],\n",
      "\n",
      "         [[0.0308]]]], device='cuda:0')]\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'cpu'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-79eedd8d222c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mimg_features\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_features\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mreturn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_features\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mtrain_feature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeature_to_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0mtest_feature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeature_to_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-79eedd8d222c>\u001b[0m in \u001b[0;36mfeature_to_array\u001b[0;34m(img_list)\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mseq_feature\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mimg_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mimg_feautures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_feature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'cpu'"
     ]
    }
   ],
   "source": [
    "def feature_to_array():\n",
    "    img_feautures = []\n",
    "    for seq_feature in img_list:\n",
    "        img_feautures.append(seq_feature.cpu().numpy())\n",
    "\n",
    "\n",
    "    img_features = np.array(img_feautures)\n",
    "    img_features=np.reshape(img_features,(20*64,-1))\n",
    "    return(img_features)\n",
    "train_featue=feature_to_array(train_list)\n",
    "test_feature=feature_to_array(test_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_features=np.concatenate([train_feature,test_feature])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(img_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "CNN_features_2d = TSNE(n_components=2, perplexity=30).fit_transform(img_features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=2)\n",
    "CNN_features_2d = pca.fit_transform(img_features)\n",
    "#principalDf = pd.DataFrame(data = principalComponents\n",
    "#             , columns = ['principal component 1', 'principal component 2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_y=np.concatenate([np.zeros(20*64),np.ones(20*64)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "cm = plt.cm.get_cmap(\"tab20\", 11)\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.scatter(CNN_features_2d[:,0], CNN_features_2d[:,1],s=20 , c=valid_y,cmap=cm,alpha=0.7)\n",
    "plt.colorbar(ticks=range(11))\n",
    "plt.clim(-0.5, 10.5)\n",
    "plt.savefig(\"CNN_tsne.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
